#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Nginx日志转换为JSONL格式并标注
将attack.log.txt和safe.log.txt转换为带标签的JSONL格式数据集
"""

import json
import os
import re

def read_logs(file_path):
    """
    读取日志文件内容
    
    Args:
        file_path (str): 日志文件路径
    
    Returns:
        list: 日志行列表
    """
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            logs = f.readlines()
        return [log.strip() for log in logs if log.strip()]
    except FileNotFoundError:
        print(f"警告: 文件 {file_path} 未找到")
        return []
    except Exception as e:
        print(f"读取文件 {file_path} 时出错: {e}")
        return []

def parse_nginx_log(log_line):
    """
    解析Nginx日志行
    
    Args:
        log_line (str): 单行日志内容
    
    Returns:
        dict: 解析后的日志字典
    """
    # Nginx默认日志格式示例:
    # 127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /index.html HTTP/1.1" 200 2326 "-" "Mozilla/5.0..."
    
    # 使用正则表达式解析日志
    log_pattern = r'(\S+) \S+ \S+ \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d+) (\d+|"-"|"") "([^"]*)" "([^"]*)"'
    
    try:
        match = re.match(log_pattern, log_line)
        if match:
            ip, timestamp, method, url, protocol, status_code, response_size, referer, user_agent = match.groups()
            
            # 处理响应大小为"-"或""的情况
            if response_size in ['-', '""', '']:
                response_size = "0"
            elif response_size.startswith('"') and response_size.endswith('"'):
                response_size = response_size[1:-1]
                
            return {
                "ip": ip,
                "timestamp": timestamp,
                "method": method,
                "url": url,
                "protocol": protocol,
                "status_code": status_code,
                "response_size": response_size,
                "referer": referer,
                "user_agent": user_agent
            }
        else:
            print(f"无法匹配日志行: {log_line[:100]}...")
            return None
    except Exception as e:
        print(f"解析日志行时出错: {e}")
        return None

def convert_logs_to_jsonl():
    """
    将日志文件转换为带标签的JSONL格式
    """
    # 读取攻击日志和正常日志
    attack_logs = read_logs("attack.log.txt")
    safe_logs = read_logs("safe.log.txt")
    
    print(f"读取到攻击日志 {len(attack_logs)} 条")
    print(f"读取到正常日志 {len(safe_logs)} 条")
    
    # 解析日志
    parsed_attack_logs = []
    parsed_safe_logs = []
    
    for log in attack_logs:
        parsed = parse_nginx_log(log)
        if parsed:
            parsed_attack_logs.append(parsed)
    
    for log in safe_logs:
        parsed = parse_nginx_log(log)
        if parsed:
            parsed_safe_logs.append(parsed)
    
    print(f"解析成功攻击日志 {len(parsed_attack_logs)} 条")
    print(f"解析成功正常日志 {len(parsed_safe_logs)} 条")
    
    # 按"正常一条、恶意一条"的交替顺序合并日志
    combined_logs = []
    max_len = max(len(parsed_attack_logs), len(parsed_safe_logs))
    
    for i in range(max_len):
        # 添加正常日志
        if i < len(parsed_safe_logs):
            safe_entry = parsed_safe_logs[i].copy()
            safe_entry["is_attack"] = 0
            combined_logs.append(safe_entry)
        
        # 添加攻击日志
        if i < len(parsed_attack_logs):
            attack_entry = parsed_attack_logs[i].copy()
            attack_entry["is_attack"] = 1
            combined_logs.append(attack_entry)
    
    # 保存为JSONL格式
    output_file = "output.jsonl"
    try:
        with open(output_file, 'w', encoding='utf-8') as f:
            for log_entry in combined_logs:
                f.write(json.dumps(log_entry, ensure_ascii=False) + '\n')
        
        print(f"成功生成标注数据集 {output_file}，共 {len(combined_logs)} 条记录")
        return True
    except Exception as e:
        print(f"保存文件 {output_file} 时出错: {e}")
        return False

if __name__ == "__main__":
    convert_logs_to_jsonl()